#' @author edvard
#' @time 2017.02.01

### cleaning ###
### Check if there have any missing value or outlier ###
summary(raw.application)
# Usually there is not specific valid value range of linear accelerometer
summary(raw.linear.accelerometer)
summary(raw.screen)
summary(raw.location)

# Removing the useless location data
raw.location <- raw.location %>% filter(double_latitude != 0.00)
# Removing the system application
raw.application <- raw.application %>% filter(is_system_app == 0)
raw.application <- raw.application %>% filter(application_name != "")
# raw.application <- raw.application %>% filter(application_name != "Xperia™中文键盘")
# raw.application <- raw.application %>% filter(application_name != "AWARE")
# raw.application <- raw.application %>% filter(application_name != "Java面试训练")

if(F == exists("application.names")){
  if(F == file.exists("cache/app_names.csv")){
    application.names <- data.frame()
    application.package <- data.frame()
  }else{
    application.names <- read.csv("cache/app_names.csv", header = T, stringsAsFactors = F)
    application.package <- read.csv("cache/app_packages.csv", header = T, stringsAsFactors = F)
  }
}

application.names <- rbind(application.names, unique(raw.application$application_name) %>% as.data.frame())
application.package <- rbind(application.package, unique(raw.application$package_name) %>% as.data.frame())
utils::write.csv(application.names, "cache/app_names.csv", row.names = F)
utils::write.csv(application.package, "cache/app_packages.csv", row.names = F)

# Transforming milliscond into second
raw.screen$timestamp <- raw.screen$timestamp/1000
raw.application$timestamp <- raw.application$timestamp/1000
# real.raw.linear.accelerometer$timestamp <- real.raw.linear.accelerometer$timestamp/1000
raw.linear.accelerometer$timestamp <- raw.linear.accelerometer$timestamp/1000
raw.location$timestamp <- raw.location$timestamp/1000


# Filter and transfer linear accelermeter into single vector
# raw.linear.accelerometer <- tbl_df(raw.linear.accelerometer)
raw.linear.accelerometer <- mutate(
  raw.linear.accelerometer, 
  square.sum = double_values_0^2 + double_values_1^2 + double_values_2^2,
  magnitude = sqrt(square.sum)
)
# Only get the magnitude over than 0.5
raw.linear.accelerometer <- filter(raw.linear.accelerometer, magnitude >= 0.5)
# raw.linear.accelerometer <- select(raw.linear.accelerometer, X_id, timestamp, magnitude)
# write.table(raw.linear.accelerometer,'cache/cleaned.linear.accelerometer.csv', row.names=F, sep = ",")

raw.screen <- AppendTimeInfo(raw.screen)
raw.application <- AppendTimeInfo(raw.application)
raw.location <- AppendTimeInfo(raw.location)
# This step alway cost a lot of time
raw.linear.accelerometer <- AppendTimeInfo(raw.linear.accelerometer)
# So, store it
write.csv(raw.linear.accelerometer, paste("cache/", user_name, "_linear_accelerometer.csv", sep = ""), row.names = F)
# raw.linear.accelerometer <- read.csv(paste("cache/", user_name, "_linear_accelerometer.csv", sep = ""), header = T)

# raw.wifi <- AppendTimeInfo(raw.wifi)

# Sorting by timestamp asc
raw.screen <- arrange(raw.screen, timestamp)
raw.application <- arrange(raw.application, timestamp)
raw.linear.accelerometer <- arrange(raw.linear.accelerometer, timestamp)
raw.location <- arrange(raw.location, timestamp)

raw.application$date <- raw.application$date %>% as.factor()
raw.location$date <- raw.location$date %>% as.factor()
raw.linear.accelerometer$date <- raw.linear.accelerometer$date %>% as.factor()
raw.screen$date <- raw.screen$date %>% as.factor()

# Segmenting the data, each segmentation represent 10 minutes data
raw.application <- raw.application %>% SegData2(60*10)
raw.location <- raw.location %>% SegData2(60*10)
raw.linear.accelerometer <- raw.linear.accelerometer %>% RepresentativeValueInLinearAccelerometer2(60*10)
raw.screen <- raw.screen %>% SegData2(60*10)


#'
#' Entering DBSCAN procedure
#'


# Calculating the inuse and standby time interval from screen data
smartphone.mode.duration = data.frame()
flag <- "inuse"
prev <- -1
ts_start <- 0
for(i in 1:dim(raw.screen)[1]){
  if(raw.screen[i, 4] != 3 && prev == -1)
    next
  
  if(raw.screen[i, 4] == 3 && prev == -1){
    prev <- 3
    ts_start <- raw.screen[i, 2]
    next
  }
  
  if(prev == 3 && raw.screen[i, 4] == 2){
    new.row <- data.frame(
      mode = flag, 
      ts_start = ts_start, 
      ts_end = raw.screen[i, 2], 
      date = raw.screen[i, 5], 
      duration = (raw.screen[i, 2] - ts_start)/60,
      time_interval = paste(as.POSIXct(ts_start, origin="1970-01-01 00:00:00"), as.POSIXct(raw.screen[i, 2], origin="1970-01-01 00:00:00"), sep = " ~ "), 
      weekday = raw.screen[i, 7]
      )
    smartphone.mode.duration <- rbind(smartphone.mode.duration, new.row)
    flag <- "standby"
    prev <- 2
    ts_start <- raw.screen[i, 2]
    next
  }
  
  if(prev == 2 && raw.screen[i, 4] == 3){
    new.row <- data.frame(
      mode = flag, 
      ts_start = ts_start, 
      ts_end = raw.screen[i, 2], 
      date = raw.screen[i, 5], 
      duration = (raw.screen[i, 2] - ts_start)/60,
      time_interval = paste(as.POSIXct(ts_start, origin="1970-01-01 00:00:00"), as.POSIXct(raw.screen[i, 2], origin="1970-01-01 00:00:00"), sep = " ~ "),
      weekday = raw.screen[i, 7]
    )
    smartphone.mode.duration <- rbind(smartphone.mode.duration, new.row)
    flag <- "inuse"
    prev <- 3
    ts_start <- raw.screen[i, 2]
    next
  }
}


# Get the common recorded date of four sensor data
tmp <- intersect(levels(raw.application$date), levels(raw.location$date))
tmp <- intersect(tmp, levels(raw.linear.accelerometer$date))
# date.list <- tmp
# The screen usgae here only for knowing the wake up and sleep time
date.list <- intersect(tmp, levels(raw.screen$date))

rm(tmp)
if(length(date.list) == 0){
  stop("The intersection of date is empty... auto exit")
}
final.data <- data.frame()
for(i in 1:length(date.list)){
  tmp.date <- date.list[i]
  tmp <- data.frame(
    date = tmp.date, 
    time_segment_no = 1:144, 
    location = "",
    application = "", 
    linear_accelerometer = "", 
    weekday = weekdays(as.POSIXct(tmp.date))
    )
  final.data <- rbind(final.data, tmp)
}

final.data$location <- as.character(final.data$location)
final.data$application <- as.character(final.data$application)
final.data$linear_accelerometer <- as.character(final.data$linear_accelerometer)

# Filling the cell according to the date and time_segment_no
for(i in 1:dim(final.data)[1]){
  query <- final.data[i, c(1,2)]
  
  tmp.loc <- coordinates %>% 
    filter(date == as.character(query[1,1]), time_segment_no == query[1,2]) %>%
    select(cluster_no)
  
  if(nrow(tmp.loc) != 0){
    tmp.loc <- table(tmp.loc)
    final.data[i, 3] <- tmp[1,1]
  }
  
  tmp.app <- raw.application %>% 
    filter(date == as.character(query[1,1]), time_segment_no == query[1,2])
  if(nrow(tmp.app) != 0){
    final.data[i, 4] <- tmp.app$application %>% paste(collapse = "=>")
  }

  tmp.lin <- raw.linear.accelerometer %>% 
    filter(date == as.character(query[1,1]), time_segment_no == query[1,2]) %>%
    select(mean_value)
  
  if(nrow(tmp.lin) != 0)
    final.data[i, 5] <- tmp.lin[1, 1]
}

cache.final.data <- final.data

final.data <- cache.final.data 
# Removing the blank row data
final.data <- final.data %>% filter(location != "" | application != "" | linear_accelerometer != "")

# Add new column according to magnitude field
final.data <- cbind(final.data, user_status = "", index = 1:dim(final.data)[1], stringsAsFactors = F)

row.walking.index <- filter(final.data, linear_accelerometer >= 0.5, linear_accelerometer <= 1.3) %>% select(index)
row.running.index <- filter(final.data, linear_accelerometer > 1.3, linear_accelerometer <= 2.0) %>% select(index)
row.transportation.index <- filter(final.data, linear_accelerometer > 2.0) %>% select(index) %>% select(index)

final.data[row.walking.index$index, 7] = "Walking"
final.data[row.running.index$index, 7] = "Running"
final.data[row.transportation.index$index, 7] = "Use Transportation"

# Add location flag
final.data$location <- paste("loc_", final.data$location, sep = "")

# Rmove column index
final.data <- final.data[, -8]

# Append workday and weekend info
final.data <- final.data %>% mutate(
  holiday = if_else(weekday %in% c("Sunday", "Saturday"), "Weekend", "Workday")
)

# Add period
final.data <- final.data %>% data.frame(period = "")
final.data$period <- final.data$period %>% as.character()
#' 1~41 Night
#' 42~65 Morning
#' 66~83 Noon
#' 84~107 Afternoon
#' 108~144 Evening
final.data[final.data$time_segment_no %in% c(1:41), 9] = "Night"
final.data[final.data$time_segment_no %in% c(42:65), 9] = "Morning"
final.data[final.data$time_segment_no %in% c(66:83), 9] = "Noon"
final.data[final.data$time_segment_no %in% c(84:107), 9] = "Afternoon"
final.data[final.data$time_segment_no %in% c(107:144), 9] = "Evening"


# Save the data with new format
write.csv(final.data, paste(user_data_path, "context_log.csv", sep = ""), row.names=F)



